{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import standard_metrics\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the dictionaries and sort for ones with similar l1 value\n",
    "\n",
    "layers = [0, 1, 2, 3, 4, 5]\n",
    "ratio = 4\n",
    "\n",
    "device = \"cuda:3\"\n",
    "\n",
    "dicts = [(layer, ratio, torch.load(f\"/mnt/ssd-cluster/bigrun0308/tied_residual_l{layer}_r{ratio}/_9/learned_dicts.pt\")) for layer in layers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_l1 = 8e-4\n",
    "\n",
    "filtered_dicts = {}\n",
    "for (layer, _, dicts_) in dicts:\n",
    "    closest_dict = None\n",
    "    closest_l1 = float(\"inf\")\n",
    "\n",
    "    for (dict, hyperparams) in dicts_:\n",
    "        l1_dist = (hyperparams[\"l1_alpha\"] - target_l1) ** 2\n",
    "        if l1_dist < closest_l1:\n",
    "            closest_dict = (dict, hyperparams)\n",
    "            closest_l1 = l1_dist\n",
    "\n",
    "    filtered_dicts[layer] = closest_dict\n",
    "    closest_dict[0].to_device(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (1, 2), (1, 3), (1, 4), (1, 5), (2, 3), (2, 4), (2, 5), (3, 4), (3, 5), (4, 5)]\n"
     ]
    }
   ],
   "source": [
    "n_features = ratio * 512\n",
    "\n",
    "covariances = {(i,j): torch.zeros(n_features, n_features, device=device) for i in layers for j in layers if i < j}\n",
    "\n",
    "print(list(covariances.keys()))\n",
    "\n",
    "means = {i: torch.zeros(n_features, device=device) for i in range(len(layers))}\n",
    "variances = {i: torch.zeros(n_features, device=device) for i in range(len(layers))}\n",
    "\n",
    "datasets = {layer: torch.load(f\"activation_data/layer_{layer}/0.pt\").to(\"cpu\") for layer in layers}\n",
    "for _, d in datasets.items():\n",
    "    d.pin_memory()\n",
    "\n",
    "dataset_len = datasets[0].shape[0]\n",
    "batch_size = 2048\n",
    "n_batches = dataset_len // batch_size\n",
    "\n",
    "import itertools\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████| 1024/1024 [00:31<00:00, 32.79it/s]\n"
     ]
    }
   ],
   "source": [
    "torch.autograd.set_grad_enabled(False)\n",
    "\n",
    "for batch_n in tqdm.tqdm(range(n_batches)):\n",
    "    samples = {l: datasets[l][batch_size*batch_n:batch_size*(batch_n+1)].to(device, dtype=torch.float32) for l in layers}\n",
    "    encoded = {l: filtered_dicts[l][0].encode(samples[l]) for l in layers}\n",
    "\n",
    "    for l in layers:\n",
    "        means[l] *= batch_n / (batch_n + 1)\n",
    "        means[l] += torch.mean(encoded[l], dim=0) / (batch_n + 1)\n",
    "\n",
    "        variances[l] *= batch_n / (batch_n + 1)\n",
    "        variances[l] += (encoded[l] - means[l]).pow(2).mean(dim=0) / (batch_n + 1)\n",
    "    \n",
    "    for (i, j) in covariances.keys():\n",
    "        covariances[(i, j)] *= batch_n / (batch_n + 1)\n",
    "        covariances[(i, j)] += (torch.einsum(\"bi,bj->ij\", encoded[i] - means[i], encoded[j] - means[j]) / batch_size) / (n_batches + 1)\n",
    "    \n",
    "    del samples, encoded\n",
    "\n",
    "correlations = {}\n",
    "\n",
    "for (i, j) in covariances.keys():\n",
    "    covariances[(i, j)] /= n_batches\n",
    "\n",
    "    correlations[(i, j)] = covariances[(i, j)] / torch.clamp(torch.sqrt(variances[i] * variances[j]), 1e-8)\n",
    "\n",
    "torch.save(means, \"means_8e-4.pkl\")\n",
    "torch.save(variances, \"variances_8e-4.pkl\")\n",
    "torch.save(covariances, \"covariances_8e-4.pkl\")\n",
    "torch.save(correlations, \"correlations_8e-4.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_sim(model, ground):\n",
    "    return torch.einsum(\"md,gd->mg\", model.get_learned_dict(), ground.get_learned_dict())\n",
    "\n",
    "def max_cosine_sim(model, ground):\n",
    "    csim_max, csim_max_idxs = cosine_sim(model, ground).max(dim=1)\n",
    "    return csim_max, csim_max_idxs\n",
    "\n",
    "cs_scores = {(i, j): cosine_sim(filtered_dicts[i][0], filtered_dicts[j][0]) for (i, j) in covariances.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n",
      "tensor(False, device='cuda:3')\n"
     ]
    }
   ],
   "source": [
    "# linear correlation between cosine sim and covariance\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import numpy as np\n",
    "\n",
    "def linear_correlation(x, y):\n",
    "    x = np.array(x.cpu()).reshape(-1, 1)\n",
    "    y = np.array(y.cpu()).reshape(-1, 1)\n",
    "    reg = LinearRegression().fit(x, y)\n",
    "    return reg.score(x, y)\n",
    "\n",
    "correlation_r_sq = np.zeros((max(layers)+1, max(layers)+1))\n",
    "\n",
    "for (i, j) in covariances.keys():\n",
    "    correlation_r_sq[i, j] = linear_correlation(correlations[i, j], cs_scores[i, j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABH6UlEQVR4nO3de3xU1bn/8e8kIZNwSbgnBCLhjkAgGiAGQUCjAdESq4DUI4FSrf4AofEGFLlUbbQWRQW52HqplQPFKvZQBDGCokTuWNHCQeQmmnBpSSBKApn1+4OTKUOGDMkkM2Tvz/v12i/NnrX2PHtPZvKwnrXXOIwxRgAAALCNkGAHAAAAgMAiAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBBAAAsBkSQAAAAJshAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBBAAAsBkSQAAAAJshAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBBAAAsBkSQAAAAJshAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBBAAAsBkSQAAAAJshAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBBAAAsBkSQAAAAJshAQQAALAZEkAAAACbIQEEAACwGRJAVNqAAQM0YMCAaj3mzJkz5XA4qvWYl2L//v1yOBz6/e9/H/DnxqVLSEjQ6NGjgx2GV8H63a1Jl/P1rorXXntNDodD+/fvr7Zjln12vPbaa9V2TCCQSAARMD/88INmzpypdevWBTuUGvXVV19p5syZ1frHBkBwLF68WHPmzAl2GEC1IwFEwPzwww+aNWuW1wRw2rRp+vHHHwMfVA346quvNGvWLBLAarR79269/PLLwQ7DKyv97qK8iyWArVu31o8//qi777478EEB1YAE0CZOnz4tl8vl9bGioqIAR1NeWFiYIiIigh0GLlNOp1N16tQJdhhe8btbM4wxF02sK/o8CxSHw6GIiAiFhoYGNQ6gqkgALyOHDx/W2LFjFRcXJ6fTqTZt2uj+++9XSUmJu80333yjYcOGqXHjxqpbt66uueYa/f3vf/c4zrp16+RwOLRkyRJNmzZNLVu2VN26dVVYWKjRo0erfv362rt3r26++WY1aNBAd911lyTJ5XJpzpw56tq1qyIiIhQTE6Nf/vKX+ve//11h3CUlJZo+fbqSk5MVHR2tevXqqV+/flq7dq27zf79+9WsWTNJ0qxZs+RwOORwODRz5kxJ3udRnT17Vo8//rjatWsnp9OphIQETZ06VcXFxR7tEhISdMstt+iTTz5R7969FRERobZt2+pPf/pTpa7/c889p9atWysyMlL9+/fXzp07y7XZtWuX7rjjDjVu3FgRERHq2bOn/va3v7kff+211zRs2DBJ0sCBA93nuW7dOmVlZalJkyYyxrjbT5gwQQ6HQy+88IJ7X35+vhwOh+bPn+/eV1xcrBkzZqh9+/ZyOp2Kj4/XI488Uu5aSNKf//xnJScnKzIyUo0bN9add96pQ4cOebQZMGCAunXrpq+++koDBw5U3bp11bJlS/3ud7+75Ov15z//Wb1791bdunXVqFEjXXfddXr//fc92rz00kvq2rWrnE6n4uLiNG7cOJ04ccKjzZ49e3T77bcrNjZWERERatWqle68804VFBS421w4J61sTtenn36qrKwsNWvWTPXq1dNtt92mo0ePlov1vffeU79+/VSvXj01aNBAQ4YM0ZdffunzHM+cOaNZs2apQ4cOioiIUJMmTdS3b1+tWbPG3cbb767D4dD48eO1bNkydenSRZGRkUpNTdUXX3whSVq4cKHat2+viIgIDRgw4JJGiw8cOKD/9//+nzp16qTIyEg1adJEw4YNK9e3MtfGGKMnnnhCrVq1Ut26dTVw4MBLui5lXC6Xnn/+eSUmJioiIkLNmjXToEGDtGXLFnebyr6PV69erZ49eyoyMlILFy6s8PNMkjZu3KhBgwYpOjpadevWVf/+/fXpp5/6jP3dd9/VkCFD3J+37dq10+OPP67S0lJ3mwEDBujvf/+7Dhw44H4vJyQkSLr4HMAPP/zQ/bvWsGFDDR06VP/85z892pT9znz99dcaPXq0GjZsqOjoaI0ZM0Y//PDDJV9/wC8Gl4XDhw+buLg4U7duXTNp0iSzYMEC89hjj5krr7zS/Pvf/zbGGJOXl2diYmJMgwYNzK9//Wvz7LPPmh49epiQkBDz9ttvu4+1du1aI8l06dLFJCUlmWeffdZkZ2eboqIik5mZaZxOp2nXrp3JzMw0CxYsMH/605+MMcb84he/MGFhYeaee+4xCxYsMI8++qipV6+e6dWrlykpKXEfv3///qZ///7un48ePWpatGhhsrKyzPz5883vfvc706lTJ1OnTh2zfft2Y4wxp06dMvPnzzeSzG233WbeeOMN88Ybb5jPP//cGGPMjBkzzIW/jpmZmUaSueOOO8y8efPMqFGjjCSTkZHh0a5169amU6dOJiYmxkydOtXMnTvXXH311cbhcJidO3dWeN337dtnJJnExESTkJBgnn76aTNr1izTuHFj06xZM5OXl+duu3PnThMdHW26dOlinn76aTN37lxz3XXXGYfD4b7+e/fuNQ888ICRZKZOneo+z7y8PPP2228bSeaLL75wH7Ps9bvjjjvc+5YtW2YkuWMvLS01N910k/t3Y+HChWb8+PEmLCzMDB061ON8nnjiCeNwOMyIESPMSy+9ZGbNmmWaNm1qEhIS3L9HZa9hXFyciY+PNxMnTjQvvfSSuf76640ks3LlygqvmTHGzJw500gyffr0Mc8884x5/vnnzc9+9jPz6KOPutuUvaZpaWnmxRdfNOPHjzehoaEev0/FxcWmTZs2Ji4uzjzxxBPmD3/4g5k1a5bp1auX2b9/v8drnJmZ6f751VdfNZLMVVddZa6//nrz4osvmgcffNCEhoaa4cOHe8T6pz/9yTgcDjNo0CDz4osvmqefftokJCSYhg0bmn379lV4nlOnTjUOh8Pcc8895uWXXzazZ882I0eONE899VS58zyfJNO9e3cTHx9vnnrqKfPUU0+Z6Ohoc8UVV5i5c+eaLl26mNmzZ5tp06aZ8PBwM3DgQJ/XfNmyZaZHjx5m+vTpZtGiRWbq1KmmUaNGpnXr1qaoqKhK12batGlGkrn55pvN3Llzzc9//nMTFxdnmjZt6nG9L2b06NFGkhk8eLCZM2eO+f3vf2+GDh1qXnzxRXebyryP27dvbxo1amQmT55sFixYYNauXVvh51lOTo4JDw83qampZvbs2ea5554z3bt3N+Hh4Wbjxo3lrsn5r3dGRoYZPny4eeaZZ8z8+fPNsGHDjCTz0EMPudu8//77JikpyTRt2tT9Xn7nnXeMMf/57Hj11Vfd7desWWPCwsJMx44dze9+9zv3+69Ro0Yez132O3PVVVeZn/70p+all14yv/jFL4wk88gjj/i87kB1IAG8TIwaNcqEhISYzZs3l3vM5XIZY4yZNGmSkWTWr1/vfuzkyZOmTZs2JiEhwZSWlhpj/pMAtm3b1vzwww8exyr7MJ48ebLH/vXr1xtJ5s033/TYv2rVqnL7L0wAz549a4qLiz36/fvf/zYxMTHm5z//uXvf0aNHjSQzY8aMcud44R/RHTt2GEnmF7/4hUe7hx56yEgyH374oXtf69atjSTz8ccfu/cdOXLEOJ1O8+CDD5Z7rvOVfYhHRkaab7/91r1/48aNRpL51a9+5d53ww03mMTERHP69Gn3PpfLZfr06WM6dOjg3leWwK1du9bjuY4cOWIkmZdeeskYY8yJEydMSEiIGTZsmImJiXG3e+CBB0zjxo3dr/sbb7xhQkJCPF53Y4xZsGCBkWQ+/fRTY4wx+/fvN6GhoebJJ5/0aPfFF1+YsLAwj/39+/c3ktzJvzHnkrHY2Fhz++23V3jN9uzZY0JCQsxtt93m/p07/3qUnWt4eLi56aabPNrMnTvXSDKvvPKKMcaY7du3G0lm2bJlFT7nxRLAtLQ093MaY8yvfvUrExoaak6cOGGMOff+aNiwobnnnns8jpeXl2eio6PL7b9Qjx49zJAhQypsc7EE0Ol0evzRX7hwoZFkYmNjTWFhoXv/lClTyiUn3lz4XjbGmNzc3HKv46Vem7LXaMiQIR7tpk6daiT5TAA//PBDI8k88MAD5R4rO15V3serVq3yaHuxzzOXy2U6dOhg0tPTPeL/4YcfTJs2bcyNN95Y7pqcf429Xc9f/vKXpm7duh7v8SFDhpjWrVuXa+stAUxKSjLNmzc3x48fd+/7/PPPTUhIiBk1apR7X9nvzPmfj8YYc9ttt5kmTZqUey6gJlACvgy4XC4tX75ct956q3r27Fnu8bLy0sqVK9W7d2/17dvX/Vj9+vV17733av/+/frqq688+mVmZioyMtLrc95///0ePy9btkzR0dG68cYbdezYMfeWnJys+vXre5RzLxQaGqrw8HD3ufzrX//S2bNn1bNnT23btu3SLsIFVq5cKUnKysry2P/ggw9KUrmyd5cuXdSvXz/3z82aNVOnTp30zTffXNLzZWRkqGXLlu6fe/furZSUFHcc//rXv/Thhx9q+PDhOnnypPv6HD9+XOnp6dqzZ48OHz5c4XM0a9ZMnTt31scffyxJ+vTTTxUaGqqHH35Y+fn52rNnjyRp/fr16tu3r/t1X7Zsma688kp17tzZ47W5/vrrJcn92rz99ttyuVwaPny4R7vY2Fh16NCh3GtYv359/dd//Zf75/DwcPXu3dvnNVu+fLlcLpemT5+ukBDPj5CymD/44AOVlJRo0qRJHm3uueceRUVFuV+/6OhoSdLq1aurVPq69957Pcqv/fr1U2lpqQ4cOCBJWrNmjU6cOKGRI0d6XJPQ0FClpKRU+HstSQ0bNtSXX37pfm0q44YbbnCXCyUpJSVFknT77berQYMG5fb7uu7nv5fPnDmj48ePq3379mrYsKHX95mva1P2GpVNQygzadKkSzq/v/71r3I4HJoxY0a5x87/zJIu/X3cpk0bpaene32+Cz/PduzYoT179uhnP/uZjh8/7n5ti4qKdMMNN+jjjz+ucJ7g+ccqe0/369dPP/zwg3bt2lXRqXv1/fffa8eOHRo9erQaN27s3t+9e3fdeOON7mtxvvvuu8/j5379+un48ePu8jZQk8KCHQCko0ePqrCwUN26dauw3YEDB9x/LM535ZVXuh8//xht2rTxepywsDC1atXKY9+ePXtUUFCg5s2be+1z5MiRCmN7/fXXNXv2bO3atUtnzpzxGYMvBw4cUEhIiNq3b++xPzY2Vg0bNnT/EStzxRVXlDtGo0aNfM5fLNOhQ4dy+zp27Ki//OUvkqSvv/5axhg99thjeuyxx7we48iRIx5JpDf9+vVz/yFYv369evbsqZ49e6px48Zav369YmJi9Pnnn+tnP/uZu8+ePXv0z3/+0z2H0tvzlrUzxng9F0nlbqJo1apVublrjRo10j/+8Y8Kz2Hv3r0KCQlRly5dLtqm7PXp1KmTx/7w8HC1bdvW/XibNm2UlZWlZ599Vm+++ab69eunn/zkJ/qv//ovd3JYkQtf90aNGkmS+3UvS9zKkuULRUVFVXj83/zmNxo6dKg6duyobt26adCgQbr77rvVvXv3SsdWdj7x8fFe9/v6Xf3xxx+VnZ2tV199VYcPH/aYS3r+fMmLPf+F16bsNbjw96VZs2buthXZu3ev4uLiPJKdC1X2fVzR58WFj5W9tpmZmRftU1BQcNFz+fLLLzVt2jR9+OGH5RIub9fTl4v9zkvnPqNXr16toqIi1atXz72/otfI1+8m4C8SQAu72Oif0+ksN3LjcrnUvHlzvfnmm177XCz5kM7dDDB69GhlZGTo4YcfVvPmzRUaGqrs7Gzt3bu36icgXfICuxe7E+/8P5L+KBtJeOihhy46QnHhHzlv+vbtq5dfflnffPON1q9fr379+snhcKhv375av3694uLi5HK5PEYzXS6XEhMT9eyzz3o9ZllC4XK55HA49N5773m9HvXr1/f4uaav2aWaPXu2Ro8erXfffVfvv/++HnjgAWVnZ+uzzz4r9w+VC/k6h7LX7Y033lBsbGy5dmFhFX8EXnfdddq7d687tj/84Q967rnntGDBAv3iF7+oUmxVve4TJkzQq6++qkmTJik1NVXR0dFyOBy68847vY50XS6vr3Tp7+OLfWZ5e6zsnJ955hklJSV57XPh73yZEydOqH///oqKitJvfvMbtWvXThEREdq2bZseffTRgN1hfDm9RrAfEsDLQLNmzRQVFeX1rtPztW7dWrt37y63v6xc0bp16yrH0K5dO33wwQe69tprK/wQ9uatt95S27Zt9fbbb3t80F9YGqrMtyW0bt1aLpdLe/bscY9wSufukD1x4oRf5+qNtxLf//7v/7pLeG3btpV0bhQtLS2twmNVdJ5lid2aNWu0efNmTZ48WdK5RGP+/PmKi4tTvXr1lJyc7O7Trl07ff7557rhhhsqPHa7du1kjFGbNm3UsWPHCmP0R7t27eRyufTVV19d9A9v2euze/du97WTzt0xvm/fvnLXMDExUYmJiZo2bZo2bNiga6+9VgsWLNATTzzhd6yS1Lx5c5+v28U0btxYY8aM0ZgxY3Tq1Cldd911mjlzps8EsLq99dZbyszM1OzZs937Tp8+Xe6u6ktV9hrt2bPH4zU6evToJY2ct2vXTqtXr9a//vWvi44C1uT7uOy1jYqKqvRru27dOh0/flxvv/22rrvuOvf+ffv2lWt7qZ9b5//OX2jXrl1q2rSpx+gfEGzMAbwMhISEKCMjQ//zP//jsXxCmbJ/Dd58883atGmTcnNz3Y8VFRVp0aJFSkhIqLAk58vw4cNVWlqqxx9/vNxjZ8+erfCPTNm/Ys//V+vGjRs94pSkunXrStIl/cG6+eabJancAqxlo2BDhgzxeYzKWL58ucccvk2bNmnjxo0aPHiwpHMJxIABA7Rw4UJ9//335fqfv7xG2Ye8t/Ns06aNWrZsqeeee05nzpzRtddeK+lcYrh371699dZbuuaaazxGpoYPH67Dhw97XQj5xx9/dK/j+NOf/lShoaGaNWtWuREEY4yOHz9+qZejQhkZGQoJCdFvfvObciMlZc+blpam8PBwvfDCCx6x/PGPf1RBQYH79SssLNTZs2c9jpGYmKiQkBCvS9xUVnp6uqKiovTb3/7WY2pCGW9LxpzvwmtWv359tW/fvlpiq6zQ0NByr+uLL77osWxJZaSlpalOnTp68cUXPY57qd96cfvtt8sYo1mzZpV77PzPLG/HrI73cXJystq1a6ff//73OnXqVLnHK3ptvX1mlZSU6KWXXirXtl69epdUEm7RooWSkpL0+uuve7z3d+7cqffff999LYDLBSOAl4nf/va3ev/999W/f3/de++9uvLKK/X9999r2bJl+uSTT9SwYUNNnjxZ//3f/63BgwfrgQceUOPGjfX6669r3759+utf/1qurFsZ/fv31y9/+UtlZ2drx44duummm1SnTh3t2bNHy5Yt0/PPP6877rjDa99bbrlFb7/9tm677TYNGTJE+/bt04IFC9SlSxePD+bIyEh16dJFS5cuVceOHdW4cWN169bN69zHHj16KDMzU4sWLXKXazZt2qTXX39dGRkZGjhwYJXP1Zv27durb9++uv/++1VcXKw5c+aoSZMmeuSRR9xt5s2bp759+yoxMVH33HOP2rZtq/z8fOXm5urbb7/V559/LklKSkpSaGionn76aRUUFMjpdOr66693z6/s16+flixZosTERPecn6uvvlr16tXT//7v/3rM/5Oku+++W3/5y1903333ae3atbr22mtVWlqqXbt26S9/+Yt73bR27drpiSee0JQpU7R//35lZGSoQYMG2rdvn9555x3de++9euihh6rlWv3617/W448/rn79+umnP/2pnE6nNm/erLi4OGVnZ6tZs2aaMmWKZs2apUGDBuknP/mJdu/erZdeekm9evVy33zy4Ycfavz48Ro2bJg6duyos2fP6o033lBoaKhuv/12v2ONiorS/Pnzdffdd+vqq6/WnXfeqWbNmungwYP6+9//rmuvvVZz5869aP8uXbpowIABSk5OVuPGjbVlyxa99dZbGj9+vN+xVdYtt9yiN954Q9HR0erSpYtyc3P1wQcfqEmTJlU6XrNmzfTQQw8pOztbt9xyi26++WZt375d7733npo2beqz/8CBA3X33XfrhRde0J49ezRo0CC5XC6tX79eAwcO1Pjx42v0fRwSEqI//OEPGjx4sLp27aoxY8aoZcuWOnz4sNauXauoqCj9z//8j9e+ffr0UaNGjZSZmakHHnhADodDb7zxhtfSa3JyspYuXaqsrCz16tVL9evX16233ur1uM8884wGDx6s1NRUjR07Vj/++KNefPFFRUdHu9c8BS4bgbzlGBU7cOCAGTVqlGnWrJlxOp2mbdu2Zty4cR5LrOzdu9fccccdpmHDhiYiIsL07t3brFixwuM4ZcsmeFtaIzMz09SrV++iMSxatMgkJyebyMhI06BBA5OYmGgeeeQR891337nbXLgMjMvlMr/97W9N69atjdPpNFdddZVZsWKFyczMLLd8woYNG0xycrIJDw/3WBLG21IaZ86cMbNmzTJt2rQxderUMfHx8WbKlCkeSzQYc275CG9LdVwYpzdlSzk888wzZvbs2SY+Pt44nU7Tr18/9xqF59u7d68ZNWqUiY2NNXXq1DEtW7Y0t9xyi3nrrbc82r388sumbdu2JjQ0tNySMPPmzTOSzP333+/RJy0tzUgyOTk55Z63pKTEPP3006Zr167G6XSaRo0ameTkZDNr1ixTUFDg0favf/2r6du3r6lXr56pV6+e6dy5sxk3bpzZvXu3x7Xp2rVruefx9ppdzCuvvGKuuuoqdzz9+/c3a9as8Wgzd+5c07lzZ1OnTh0TExNj7r//fo/1CL/55hvz85//3LRr185ERESYxo0bm4EDB5oPPvjA4zgXWwbmwmWTyn73L1yCZ+3atSY9Pd1ER0ebiIgI065dOzN69GizZcuWCs/xiSeeML179zYNGzY0kZGRpnPnzubJJ5/0WBfzYsvAjBs3zmPf+b9r3mL2tRTOv//9bzNmzBjTtGlTU79+fZOenm527drl17UpLS01s2bNMi1atDCRkZFmwIABZufOneWOeTFnz541zzzzjOncubMJDw83zZo1M4MHDzZbt251t/H3fezr+mzfvt389Kc/NU2aNDFOp9O0bt3aDB8+3ON95G0ZmE8//dRcc801JjIy0sTFxZlHHnnErF69utw1OnXqlPnZz35mGjZsaCS53x/eloExxpgPPvjAXHvttSYyMtJERUWZW2+91Xz11Vcebcp+Z44ePeqx31ucQE1xGMNsUwAAADthDiAAAIDNkAACAADYDAkgAACAzZAAAgAA2AwJIAAAgM2QAAIAANgMCSAAAIDN8E0gtYTL5dJ3332nBg0aVOo7dQEAlwdjjE6ePKm4uDi/vrnJl9OnT6ukpMTv44SHhysiIqIaIsLliASwlvjuu+8UHx8f7DAAAH46dOiQWrVqVSPHPn36tNq0rq+8I1X7jujzxcbGat++fSSBFkUCWEs0aNBAktRXNytMdYIcTWAd/vOVwQ4h4Ep3NQh2CEHR6J/2/GKiqG9O+W5kMaFHTgQ7hIA76yrRuu9fcX+e14SSkhLlHSnVvq2tFdWg6qOMhSddapN8QCUlJSSAFkUCWEuUlX3DVEdhDnslgKF1ncEOIeCMTT9ww+rYMwEMCz0T7BACLjTEfu/rMoGYxlOv/rmtqkrt+Va0FW4CAQAAsBlGAAEAsBiXjFyq+jCeP31RO5AAAgBgMS655PKzP6yNEjAAAIDNMAIIAIDFlBqjUlP1Mq4/fVE7kAACAGAxzAGEL5SAAQAAbIYRQAAALMYlo1JGAFEBEkAAACyGEjB8oQQMAABgM4wAAgBgMdwFDF9IAAEAsBjX/23+9Ie1kQACAGAxpX7eBOJPX9QOzAEEAACwGUYAAQCwmFJzbvOnP6yNBBAAAIthDiB8oQQMAABgM4wAAgBgMS45VCqHX/1hbSSAAABYjMuc2/zpD2ujBAwAAGAzjAACAGAxpX6WgP3pi9qBBBAAAIshAYQvlIABAEC1mDdvnhISEhQREaGUlBRt2rSpwvbLli1T586dFRERocTERK1cudLj8dGjR8vhcHhsgwYNqslTsA0SwACr7JsDAIDKchmH31tlLV26VFlZWZoxY4a2bdumHj16KD09XUeOHPHafsOGDRo5cqTGjh2r7du3KyMjQxkZGdq5c6dHu0GDBun77793b//93/9dpWsCTySAAVTZNwcAAFVRVgL2Z6usZ599Vvfcc4/GjBmjLl26aMGCBapbt65eeeUVr+2ff/55DRo0SA8//LCuvPJKPf7447r66qs1d+5cj3ZOp1OxsbHurVGjRlW6JvBEAhhAlX1zAABQFaUK8XuTpMLCQo+tuLjY6/OVlJRo69atSktLc+8LCQlRWlqacnNzvfbJzc31aC9J6enp5dqvW7dOzZs3V6dOnXT//ffr+PHj/lwa/B8SwACp7JujuLi43BsPAIBAio+PV3R0tHvLzs722u7YsWMqLS1VTEyMx/6YmBjl5eV57ZOXl+ez/aBBg/SnP/1JOTk5evrpp/XRRx9p8ODBKi0t9fPMwF3AAVLRm2PXrl3l2mdnZ2vWrFmBCg8AYCGmivP4zu8vSYcOHVJUVJR7v9Pp9Du2yrjzzjvd/5+YmKju3burXbt2WrdunW644YaAxmI1jABepqZMmaKCggL3dujQoWCHBACoJaprDmBUVJTHdrEEsGnTpgoNDVV+fr7H/vz8fMXGxnrtExsbW6n2ktS2bVs1bdpUX3/9dWUuB7wgAQyQyr45nE5nuTceAACXo/DwcCUnJysnJ8e9z+VyKScnR6mpqV77pKamerSXpDVr1ly0vSR9++23On78uFq0aFE9gdsYCWCAVOXNAQBAVZSaEL+3ysrKytLLL7+s119/Xf/85z91//33q6ioSGPGjJEkjRo1SlOmTHG3nzhxolatWqXZs2dr165dmjlzprZs2aLx48dLkk6dOqWHH35Yn332mfbv36+cnBwNHTpU7du3V3p6evVcKBtjDmAAZWVlKTMzUz179lTv3r01Z84cjzcHAADVwSWHXH6M8bhkKt1nxIgROnr0qKZPn668vDwlJSVp1apV7rnvBw8eVEjIf2Lq06ePFi9erGnTpmnq1Knq0KGDli9frm7dukmSQkND9Y9//EOvv/66Tpw4obi4ON100016/PHHAz4X0YpIAAPI15sDAIDabPz48e4RvAutW7eu3L5hw4Zp2LBhXttHRkZq9erV1RkezkMCGGAVvTkAAKgOfBcwfCEBBADAYqo6j+8//StfAkbtwk0gAAAANsMIIAAAFnPuJpCql3H96YvagQQQAACLcZ33fb5V608J2OpIAAEAsBjmAMIX5gACAADYDCOAAABYjEshAV8IGrULCSAAABZTahwqNX6sA+hHX9QOlIABAABshhFAAAAsptTPu4BLKQFbHgkgAAAW4zIhcvlxF7CLu4AtjxIwAACAzTACCACAxVAChi8kgAAAWIxL/t3J66q+UHCZogQMAABgM4wAAgBgMf4vBM34kNWRAAIAYDH+fxcwCaDVkQACAGAxLjnkkj9zAPkmEKsjxQcAALAZRgABALAYSsDwhQQQAACL8X8dQBJAq+MVBgAAsBlGAAEAsBiXccjlz0LQfvRF7UACCACAxbj8LAGzDqD1kQDistfq9i+DHULArf5uR7BDCIp2H44JdgjBsaJBsCMIuKgQ+yUYpaWnpcPBjgI4hwQQAACLcZkQufy4k9efvqgdSAABALCYUjlU6sdizv70Re1Aig8AAGAzjAACAGAxlIDhCwkgAAAWUyr/yril1RcKLlMkgAAAWAwjgPCFVxgAAMBmGAEEAMBiSk2ISv0YxfOnL2oHEkAAACzGyCGXH3MADcvAWB4pPgAAgM0wAggAgMVQAoYvJIAAAFiMyzjkMlUv4/rTF7UDKT4AAIDNMAIIAIDFlCpEpX6M8fjTF7UDCSAAABZDCRi+kOIDAADYDCOAAABYjEshcvkxxuNPX9QOJIAAAFhMqXGo1I8yrj99UTuQAAIAYDHMAYQvjPECAADYDCOAAABYjDEhcvnxbR6GbwKxPBJAAAAsplQOlcqPOYB+9EXtQIoPAABgM4wAAgBgMS7j340cLlONweCyRAIIAIDFuPycA+hPX9QOvMIAAAA2QwIYIB9//LFuvfVWxcXFyeFwaPny5cEOCQBgUS45/N5gbSSAAVJUVKQePXpo3rx5wQ4FAGBxZd8E4s8Ga2MOYIAMHjxYgwcPDnYYAAAAJICXq+LiYhUXF7t/LiwsDGI0AIDahJtA4Auv8GUqOztb0dHR7i0+Pj7YIQEAagmXHO7vA67SxhxAyyMBvExNmTJFBQUF7u3QoUPBDgkAUEsYP28AMVVMAOfNm6eEhARFREQoJSVFmzZtqrD9smXL1LlzZ0VERCgxMVErV668aNv77rtPDodDc+bMqVJs8EQCeJlyOp2Kiory2AAAuFwtXbpUWVlZmjFjhrZt26YePXooPT1dR44c8dp+w4YNGjlypMaOHavt27crIyNDGRkZ2rlzZ7m277zzjj777DPFxcXV9GnYBgkgAAAW41f59/+2ynr22Wd1zz33aMyYMerSpYsWLFigunXr6pVXXvHa/vnnn9egQYP08MMP68orr9Tjjz+uq6++WnPnzvVod/jwYU2YMEFvvvmm6tSpU6XrgfJIAAPk1KlT2rFjh3bs2CFJ2rdvn3bs2KGDBw8GNzAAgOWU3QTizyaduwHx/O38mxPPV1JSoq1btyotLc29LyQkRGlpacrNzfXaJzc316O9JKWnp3u0d7lcuvvuu/Xwww+ra9eu/l4WnIcEMEC2bNmiq666SldddZUkKSsrS1dddZWmT58e5MgAAPAuPj7e44bE7Oxsr+2OHTum0tJSxcTEeOyPiYlRXl6e1z55eXk+2z/99NMKCwvTAw884OeZ4EIsAxMgAwYMkDF8uzYAoOZVtYx7fn9JOnTokMccdKfT6Xdsl2rr1q16/vnntW3bNjkc3JVc3RgBBADAYqrrq+AuvBnxYglg06ZNFRoaqvz8fI/9+fn5io2N9donNja2wvbr16/XkSNHdMUVVygsLExhYWE6cOCAHnzwQSUkJPh5hUACCAAA/BIeHq7k5GTl5OS497lcLuXk5Cg1NdVrn9TUVI/2krRmzRp3+7vvvlv/+Mc/3PPnd+zYobi4OD388MNavXp1zZ2MTVACBgDAYqqrBFwZWVlZyszMVM+ePdW7d2/NmTNHRUVFGjNmjCRp1KhRatmypXse4cSJE9W/f3/Nnj1bQ4YM0ZIlS7RlyxYtWrRIktSkSRM1adLE4znq1Kmj2NhYderUqcrnhnNIAAEAsJhgJIAjRozQ0aNHNX36dOXl5SkpKUmrVq1y3+hx8OBBhYT8p/DYp08fLV68WNOmTdPUqVPVoUMHLV++XN26daty3Lh0JIAAAKBajB8/XuPHj/f62Lp168rtGzZsmIYNG3bJx9+/f38VI8OFSAABALCYYIwAonYhAQQAwGJIAOELCSAAABZjJPdSLlXtD2tjGRgAAACbYQQQAACLoQQMX0gAAQCwGBJA+EIJGAAAwGYYAQQAwGIYAYQvJIAAAFgMCSB8oQQMAABgM4wAAgBgMcY4ZPwYxfOnL2oHEkAAACzGJYdfC0H70xe1AyVgAAAAm2EEEAAAi+EmEPhCAggAgMUwBxC+kAACAGAxjADCF+YAAgAA2AwjgAAAWAwlYPhCAghchtLjkoIdQlC01/Zgh4AAMcEOIAiMORPA5/KvBEwCaH2UgAEAAGyGEUAAACzGSDJ+DLPacYTWbkgAAQCwGJcccvBNIKgAJWAAAACbYQQQAACL4S5g+EICCACAxbiMQw4WgkYFKAEDAADYDCOAAABYjDF+3gXMbcCWRwIIAIDFMAcQvpAAAgBgMSSA8IU5gAAAADbDCCAAABbDXcDwhQQQAACL4SYQ+EIJGAAAwGYYAQQAwGLOjQD6cxNINQaDyxIJIAAAFsNdwPCFEjAAAIDNMAIIAIDFmP/b/OkPayMBBADAYigBwxdKwAAAADbDCCAAAFZDDRg+kAACAGA1fpaARQnY8kgAAQCwGL4JBL4wBxAAAMBmGAEEAMBiuAsYvpAAAgBgNcbh3zw+EkDLowQcINnZ2erVq5caNGig5s2bKyMjQ7t37w52WAAAwIZIAAPko48+0rhx4/TZZ59pzZo1OnPmjG666SYVFRUFOzQAgMWU3QTizwZrowQcIKtWrfL4+bXXXlPz5s21detWXXfddUGKCgBgSawDCB8YAQySgoICSVLjxo2DHAkAALAbRgCDwOVyadKkSbr22mvVrVs3r22Ki4tVXFzs/rmwsDBQ4QEAajnuAoYvjAAGwbhx47Rz504tWbLkom2ys7MVHR3t3uLj4wMYIQCg1jN+bLA8EsAAGz9+vFasWKG1a9eqVatWF203ZcoUFRQUuLdDhw4FMEoAAGBllIADxBijCRMm6J133tG6devUpk2bCts7nU45nc4ARQcAsBJKwPCFBDBAxo0bp8WLF+vdd99VgwYNlJeXJ0mKjo5WZGRkkKMDAFgKdwHDB0rAATJ//nwVFBRowIABatGihXtbunRpsEMDAFiOoxo2WBkjgAFiWFUTAABcJkgAAQCwGkrA8IESMAAAVuPPEjB+JI/z5s1TQkKCIiIilJKSok2bNlXYftmyZercubMiIiKUmJiolStXejw+c+ZMde7cWfXq1VOjRo2UlpamjRs3Vi04eCABBAAAflu6dKmysrI0Y8YMbdu2TT169FB6erqOHDnitf2GDRs0cuRIjR07Vtu3b1dGRoYyMjK0c+dOd5uOHTtq7ty5+uKLL/TJJ58oISFBN910k44ePRqo07Ish2FyWq1QWFio6OhoDdBQhTnqBDscAEAlnTVntE7vqqCgQFFRUTXyHGV/K+LnzVJIZESVj+P68bQOjZtRqVhTUlLUq1cvzZ0799wxXC7Fx8drwoQJmjx5crn2I0aMUFFRkVasWOHed8011ygpKUkLFizw+hxl5/fBBx/ohhtuqMKZoQwjgAAAWIwx/m/SuYTr/O38ryg9X0lJibZu3aq0tDT3vpCQEKWlpSk3N9drn9zcXI/2kpSenn7R9iUlJVq0aJGio6PVo0ePKlwVnI8EEAAAeBUfH+/xtaTZ2dle2x07dkylpaWKiYnx2B8TE+Ne9/ZCeXl5l9R+xYoVql+/viIiIvTcc89pzZo1atq0qR9nBYm7gAEAsJ5qugv40KFDHiXgYHxD1cCBA7Vjxw4dO3ZML7/8soYPH66NGzeqefPmAY/FShgBBADAaozD/01SVFSUx3axBLBp06YKDQ1Vfn6+x/78/HzFxsZ67RMbG3tJ7evVq6f27dvrmmuu0R//+EeFhYXpj3/8Y1WvDP4PCSAAAPBLeHi4kpOTlZOT497ncrmUk5Oj1NRUr31SU1M92kvSmjVrLtr+/ONebC4iLh0lYAAALMZhzm3+9K+srKwsZWZmqmfPnurdu7fmzJmjoqIijRkzRpI0atQotWzZ0j2PcOLEierfv79mz56tIUOGaMmSJdqyZYsWLVokSSoqKtKTTz6pn/zkJ2rRooWOHTumefPm6fDhwxo2bFjVTw6SSAABALCeIHwTyIgRI3T06FFNnz5deXl5SkpK0qpVq9w3ehw8eFAhIf8pPPbp00eLFy/WtGnTNHXqVHXo0EHLly9Xt27dJEmhoaHatWuXXn/9dR07dkxNmjRRr169tH79enXt2tWPk4PEOoC1BusAAkDtFtB1AJ973P91AH/1WI3GiuBiDiAAAIDNUAIGAMBqglACRu1CAggAgNWQAMIHSsAAAAA2wwggAABWwwggfCABBADAas77No8q94elUQIGAACwGUYAAQCwmGB8EwhqFxJAAACshjmA8IESMAAAgM2QAAIAANgMJWAAACzGIT/nAFZbJLhckQACAGA1LAMDHygBAwAA2AwjgAAAWA13AcMHEkAAAKyGBBA+UAIGAACwGUYAAQCwGL4JBL6QAAIAYDWUgOEDJWAAAACbYQQQAACrYQQQPpAAAgBgMcwBhC+UgAEAAGyGEUAAAKyGr4KDDySAAABYDXMA4QMJIAAAFsMcQPjCHEAAAACbYQQQAACroQQMH0gAAQCwGj9LwCSA1kcJGAAAwGYYAQQAwGooAcMHEkAAAKyGBBA+UAIGAACwGUYAAQCwGNYBhC+MAAIAANgMCSAAAIDNUAIGAMBquAkEPpAAAgBgMcwBhC8kgAAAWBFJHCrAHMAAmT9/vrp3766oqChFRUUpNTVV7733XrDDAgAANkQCGCCtWrXSU089pa1bt2rLli26/vrrNXToUH355ZfBDg0AYDWmGjZYGiXgALn11ls9fn7yySc1f/58ffbZZ+ratWuQogIAWBFzAOELCWAQlJaWatmyZSoqKlJqaqrXNsXFxSouLnb/XFhYGKjwAACAxVECDqAvvvhC9evXl9Pp1H333ad33nlHXbp08do2Oztb0dHR7i0+Pj7A0QIAai1KwPCBBDCAOnXqpB07dmjjxo26//77lZmZqa+++spr2ylTpqigoMC9HTp0KMDRAgBqq7ISsD8brI0ScACFh4erffv2kqTk5GRt3rxZzz//vBYuXFiurdPplNPpDHSIAADABkgAg8jlcnnM8wMAoFrwTSDwgQQwQKZMmaLBgwfriiuu0MmTJ7V48WKtW7dOq1evDnZoAACrIQGEDySAAXLkyBGNGjVK33//vaKjo9W9e3etXr1aN954Y7BDAwBYDMvAwBcSwAD54x//GOwQAAAAJJEAAgBgPZSA4QMJIAAAVkMCCB9YBxAAAMBmGAEEAMBiuAkEvpAAAgBgNZSA4QMlYAAAAJthBBAAAIuhBAxfGAEEAMBqTDVsVTBv3jwlJCQoIiJCKSkp2rRpU4Xtly1bps6dOysiIkKJiYlauXKl+7EzZ87o0UcfVWJiourVq6e4uDiNGjVK3333XdWCgwcSQAAA4LelS5cqKytLM2bM0LZt29SjRw+lp6fryJEjXttv2LBBI0eO1NixY7V9+3ZlZGQoIyNDO3fulCT98MMP2rZtmx577DFt27ZNb7/9tnbv3q2f/OQngTwty3IYYxjorQUKCwsVHR2tARqqMEedYIcDAKiks+aM1uldFRQUKCoqqkaeo+xvxZX/77cKdUZU+Tilxaf1z5emVirWlJQU9erVS3PnzpUkuVwuxcfHa8KECZo8eXK59iNGjFBRUZFWrFjh3nfNNdcoKSlJCxYs8PocmzdvVu/evXXgwAFdccUVVTgzlGEEEAAAi3FUwyadSyjP34qLi70+X0lJibZu3aq0tDT3vpCQEKWlpSk3N9drn9zcXI/2kpSenn7R9pJUUFAgh8Ohhg0bVnj+8I0EEAAAq6mmOYDx8fGKjo52b9nZ2V6f7tixYyotLVVMTIzH/piYGOXl5Xntk5eXV6n2p0+f1qOPPqqRI0fW2AiqnXAXMAAA8OrQoUMeyZbT6QxKHGfOnNHw4cNljNH8+fODEoPVkAACAGAx1bUMTFRU1CWNtjVt2lShoaHKz8/32J+fn6/Y2FivfWJjYy+pfVnyd+DAAX344YeM/lUTSsAAAFhNgJeBCQ8PV3JysnJyctz7XC6XcnJylJqa6rVPamqqR3tJWrNmjUf7suRvz549+uCDD9SkSZPKBYaLYgQQAAD4LSsrS5mZmerZs6d69+6tOXPmqKioSGPGjJEkjRo1Si1btnTPI5w4caL69++v2bNna8iQIVqyZIm2bNmiRYsWSTqX/N1xxx3atm2bVqxYodLSUvf8wMaNGys8PDw4J2oRJIAAAFhRgBd5GzFihI4eParp06crLy9PSUlJWrVqlftGj4MHDyok5D+Fxz59+mjx4sWaNm2apk6dqg4dOmj58uXq1q2bJOnw4cP629/+JklKSkryeK61a9dqwIABATkvq2IdwFqCdQABoHYL5DqA3e79rULD/VgHsOS0di6q3DqAqF2YAwgAAGAzlIABALAaP77P190flkYCCACAxVTXMjCwLkrAAAAANsMIIAAAVkMJGD6QAAIAYDGUgOELCSAAAFbDCCB8YA4gAACAzTACCACA1TACCB9IAAEAsBjmAMIXSsAAAAA2wwggAABWQwkYPpAAAgBgMQ5j5DBVz+L86YvagRIwAACAzTACCACA1VAChg8kgAAAWAx3AcMXSsAAAAA2wwggAABWQwkYPpAAAgBgMZSA4QsJIAAAVsMIIHxgDiAAAIDNMAIIAIDFUAKGLySAAABYDSVg+EAJGAAAwGYYAQQAwIIo46IiJIAAAFiNMec2f/rD0igBAwAA2AwjgAAAWAx3AcMXEkAAAKyGu4DhAyVgAAAAm2EEEAAAi3G4zm3+9Ie1kQACAGA1lIDhAyXgIHjqqafkcDg0adKkYIcCALCgsptA/NlgbSSAAbZ582YtXLhQ3bt3D3YoAADApkgAA+jUqVO666679PLLL6tRo0bBDgcAYFVlC0H7s8HSSAADaNy4cRoyZIjS0tKCHQoAwMIoAcMXbgIJkCVLlmjbtm3avHnzJbUvLi5WcXGx++fCwsKaCg0AANgMI4ABcOjQIU2cOFFvvvmmIiIiLqlPdna2oqOj3Vt8fHwNRwkAsAxTDRssjQQwALZu3aojR47o6quvVlhYmMLCwvTRRx/phRdeUFhYmEpLS8v1mTJligoKCtzboUOHghA5AKA2ogQMXygBB8ANN9ygL774wmPfmDFj1LlzZz366KMKDQ0t18fpdMrpdAYqRAAAYCMkgAHQoEEDdevWzWNfvXr11KRJk3L7AQDwm7938nIXsOWRAAIAYDH+lnEpAVsfCWCQrFu3LtghAAAAmyIBBADAavguYPhAAggAgMVQAoYvJIAAAFiNy5zb/OkPS2MdQAAAAJthBBAAAKthDiB8IAEEAMBiHPJzDmC1RYLLFSVgAAAAm2EEEAAAq+GbQOADCSAAABbDMjDwhRIwAACAzTACCACA1XAXMHwgAQQAwGIcxsjhxzw+f/qidqAEDAAAYDOMAAIAYDWu/9v86Q9LYwQQAACLKSsB+7NVxbx585SQkKCIiAilpKRo06ZNFbZftmyZOnfurIiICCUmJmrlypUej7/99tu66aab1KRJEzkcDu3YsaNKcaE8EkAAAKzGVMNWSUuXLlVWVpZmzJihbdu2qUePHkpPT9eRI0e8tt+wYYNGjhypsWPHavv27crIyFBGRoZ27tzpblNUVKS+ffvq6aefrnxAqJDDGGZ61gaFhYWKjo7WAA1VmKNOsMMBAFTSWXNG6/SuCgoKFBUVVSPPUfa34rq+0xUWFlHl45w9e1off/KbSsWakpKiXr16ae7cuZIkl8ul+Ph4TZgwQZMnTy7XfsSIESoqKtKKFSvc+6655holJSVpwYIFHm3379+vNm3aaPv27UpKSqryeeE/GAEEAMBqyr4JxJ9N5xLK87fi4mKvT1dSUqKtW7cqLS3NvS8kJERpaWnKzc312ic3N9ejvSSlp6dftD2qFwkgAAAWU/ZNIP5skhQfH6/o6Gj3lp2d7fX5jh07ptLSUsXExHjsj4mJUV5entc+eXl5lWqP6sVdwAAAwKtDhw55lICdTmcQo0F1IgEEAMBqzivjVrm/pKioqEuaA9i0aVOFhoYqPz/fY39+fr5iY2O99omNja1Ue1QvSsAAAFiMw+X/Vhnh4eFKTk5WTk6Oe5/L5VJOTo5SU1O99klNTfVoL0lr1qy5aHtUL0YAAQCA37KyspSZmamePXuqd+/emjNnjoqKijRmzBhJ0qhRo9SyZUv3PMKJEyeqf//+mj17toYMGaIlS5Zoy5YtWrRokfuY//rXv3Tw4EF99913kqTdu3dLOjd6yEihf0gAAQCwmmoqAVfGiBEjdPToUU2fPl15eXlKSkrSqlWr3Dd6HDx4UCEh/yk89unTR4sXL9a0adM0depUdejQQcuXL1e3bt3cbf72t7+5E0hJuvPOOyVJM2bM0MyZM6t4cpBYB7DWYB1AAKjdArkO4IBev/Z7HcB1m5+s0VgRXMwBBAAAsBlKwAAAWIw/3+db1h/WRgIIAIDVBGEOIGoXEkAAAKzGSKrkUi7l+sPSmAMIAABgM4wAAgBgMcwBhC8kgAAAWI2Rn3MAqy0SXKYoAQMAANgMI4AAAFgNdwHDBxJAAACsxiXJ4Wd/WBolYAAAAJthBBAAAIvhLmD4QgIIAIDVMAcQPlACBgAAsBlGAAEAsBpGAOEDCSAAAFZDAggfSAABALAaloGBD8wBBAAAsBlGAAEAsBiWgYEvJIAAAFgNcwDhAyVgAAAAm2EEEAAAq3EZyeHHKJ6LEUCrIwEEAMBqKAHDB0rAAAAANsMIIAAAluPnCKAYAbQ6EkAAAKyGEjB8oAQcIDNnzpTD4fDYOnfuHOywAACADTECGEBdu3bVBx984P45LIzLDwCoAS4jv8q43AVseWQgARQWFqbY2NhghwEAsDrjOrf50x+WRgk4gPbs2aO4uDi1bdtWd911lw4ePHjRtsXFxSosLPTYAAC4JGVzAP3ZYGkkgAGSkpKi1157TatWrdL8+fO1b98+9evXTydPnvTaPjs7W9HR0e4tPj4+wBEDAACrchhDmh8MJ06cUOvWrfXss89q7Nix5R4vLi5WcXGx++fCwkLFx8drgIYqzFEnkKECAKrBWXNG6/SuCgoKFBUVVSPPUVhYqOjoaKW1vE9hIc4qH+esq1gfHF5Qo7EiuJgDGCQNGzZUx44d9fXXX3t93Ol0yums+psXAGBjLAMDHygBB8mpU6e0d+9etWjRItihAAAAmyEBDJCHHnpIH330kfbv368NGzbotttuU2hoqEaOHBns0AAAVmPk500gwT4B1DRKwAHy7bffauTIkTp+/LiaNWumvn376rPPPlOzZs2CHRoAwGooAcMHEsAAWbJkSbBDAAAAkEQCCACA9bhckvxYzNnFQtBWRwIIAIDVUAKGD9wEAgAAYDOMAAIAYDWMAMIHEkAAAKzGZeTXWi4uEkCrIwEEAMBijHHJmKrfyOFPX9QOzAEEAACwGUYAAQCwGmP8K+MyB9DySAABALAa4+ccQBJAy6MEDAAAYDOMAAIAYDUul+Tw40YObgKxPBJAAACshhIwfKAEDAAAYDOMAAIAYDHG5ZLxowTMOoDWRwIIAIDVUAKGD5SAAQAAbIYRQAAArMZlJAcjgLg4EkAAAKzGGEn+LANDAmh1JIAAAFiMcRkZP0YADQmg5TEHEAAAwGZIAAEAsBrj8n+rgnnz5ikhIUERERFKSUnRpk2bKmy/bNkyde7cWREREUpMTNTKlSs9T8MYTZ8+XS1atFBkZKTS0tK0Z8+eKsUGTySAAABYjHEZv7fKWrp0qbKysjRjxgxt27ZNPXr0UHp6uo4cOeK1/YYNGzRy5EiNHTtW27dvV0ZGhjIyMrRz5053m9/97nd64YUXtGDBAm3cuFH16tVTenq6Tp8+XeVrg3MchkJ/rVBYWKjo6GgN0FCFOeoEOxwAQCWdNWe0Tu+qoKBAUVFRNfIc7r8Vjtv8+ltx1pzROvNOpWJNSUlRr169NHfuXEmSy+VSfHy8JkyYoMmTJ5drP2LECBUVFWnFihXufddcc42SkpK0YMECGWMUFxenBx98UA899JAkqaCgQDExMXrttdd05513Vvn8wE0gtUZZnn5WZ/xa2xMAEBxndUZSYG6wOGuKq1zGlf4Ta2Fhocd+p9Mpp9NZrn1JSYm2bt2qKVOmuPeFhIQoLS1Nubm5Xp8jNzdXWVlZHvvS09O1fPlySdK+ffuUl5entLQ09+PR0dFKSUlRbm4uCaCfSABriZMnT0qSPtFKHy0BAJezkydPKjo6ukaOHR4ertjYWH2S5//fivr16ys+Pt5j34wZMzRz5sxybY8dO6bS0lLFxMR47I+JidGuXbu8Hj8vL89r+7y8PPfjZfsu1gZVRwJYS8TFxenQoUNq0KCBHA5HQJ+7sLBQ8fHxOnToUI2VLS5HdjxvO56zZM/ztuM5S8E9b2OMTp48qbi4uBp7joiICO3bt08lJSV+H8sYU+7vjbfRP9ROJIC1REhIiFq1ahXUGKKiomz1h6KMHc/bjucs2fO87XjOUvDOu6ZG/s4XERGhiIiIGn+e8zVt2lShoaHKz8/32J+fn6/Y2FivfWJjYytsX/bf/Px8tWjRwqNNUlJSNUZvT9wFDAAA/BIeHq7k5GTl5OS497lcLuXk5Cg1NdVrn9TUVI/2krRmzRp3+zZt2ig2NtajTWFhoTZu3HjRY+LSMQIIAAD8lpWVpczMTPXs2VO9e/fWnDlzVFRUpDFjxkiSRo0apZYtWyo7O1uSNHHiRPXv31+zZ8/WkCFDtGTJEm3ZskWLFi2SJDkcDk2aNElPPPGEOnTooDZt2uixxx5TXFycMjIygnWalkECCJ+cTqdmzJhhu7kfdjxvO56zZM/ztuM5S/Y970AYMWKEjh49qunTpysvL09JSUlatWqV+yaOgwcPKiTkP4XHPn36aPHixZo2bZqmTp2qDh06aPny5erWrZu7zSOPPKKioiLde++9OnHihPr27atVq1YFvMRtRawDCAAAYDPMAQQAALAZEkAAAACbIQEEAACwGRJAAAAAmyEBhE/z5s1TQkKCIiIilJKSok2bNgU7pBr18ccf69Zbb1VcXJwcDof7eymtLDs7W7169VKDBg3UvHlzZWRkaPfu3cEOq0bNnz9f3bt3dy8InJqaqvfeey/YYQXcU0895V5uw8pmzpwph8PhsXXu3DnYYQFBQwKICi1dulRZWVmaMWOGtm3bph49eig9PV1HjhwJdmg1pqioSD169NC8efOCHUrAfPTRRxo3bpw+++wzrVmzRmfOnNFNN92koqKiYIdWY1q1aqWnnnpKW7du1ZYtW3T99ddr6NCh+vLLL4MdWsBs3rxZCxcuVPfu3YMdSkB07dpV33//vXv75JNPgh0SEDQsA4MKpaSkqFevXpo7d66kcyu7x8fHa8KECZo8eXKQo6t5DodD77zzju0WHT169KiaN2+ujz76SNddd12wwwmYxo0b65lnntHYsWODHUqNO3XqlK6++mq99NJLeuKJJ5SUlKQ5c+YEO6waM3PmTC1fvlw7duwIdijAZYERQFxUSUmJtm7dqrS0NPe+kJAQpaWlKTc3N4iRoaYVFBRIOpcQ2UFpaamWLFmioqIi23zF1Lhx4zRkyBCP97fV7dmzR3FxcWrbtq3uuusuHTx4MNghAUHDN4Hgoo4dO6bS0lL3Ku5lYmJitGvXriBFhZrmcrk0adIkXXvttR4r8lvRF198odTUVJ0+fVr169fXO++8oy5dugQ7rBq3ZMkSbdu2TZs3bw52KAGTkpKi1157TZ06ddL333+vWbNmqV+/ftq5c6caNGgQ7PCAgCMBBOBh3Lhx2rlzpy3mR3Xq1Ek7duxQQUGB3nrrLWVmZuqjjz6ydBJ46NAhTZw4UWvWrLHV12kNHjzY/f/du3dXSkqKWrdurb/85S+2KPkDFyIBxEU1bdpUoaGhys/P99ifn5+v2NjYIEWFmjR+/HitWLFCH3/8sVq1ahXscGpceHi42rdvL0lKTk7W5s2b9fzzz2vhwoVBjqzmbN26VUeOHNHVV1/t3ldaWqqPP/5Yc+fOVXFxsUJDQ4MYYWA0bNhQHTt21Ndffx3sUICgYA4gLio8PFzJycnKyclx73O5XMrJybHNPCm7MMZo/Pjxeuedd/Thhx+qTZs2wQ4pKFwul4qLi4MdRo264YYb9MUXX2jHjh3urWfPnrrrrru0Y8cOWyR/0rmbYPbu3asWLVoEOxQgKBgBRIWysrKUmZmpnj17qnfv3pozZ46Kioo0ZsyYYIdWY06dOuUxKrBv3z7t2LFDjRs31hVXXBHEyGrOuHHjtHjxYr377rtq0KCB8vLyJEnR0dGKjIwMcnQ1Y8qUKRo8eLCuuOIKnTx5UosXL9a6deu0evXqYIdWoxo0aFBubme9evXUpEkTS8/5fOihh3TrrbeqdevW+u677zRjxgyFhoZq5MiRwQ4NCAoSQFRoxIgROnr0qKZPn668vDwlJSVp1apV5W4MsZItW7Zo4MCB7p+zsrIkSZmZmXrttdeCFFXNmj9/viRpwIABHvtfffVVjR49OvABBcCRI0c0atQoff/994qOjlb37t21evVq3XjjjcEODTXg22+/1ciRI3X8+HE1a9ZMffv21WeffaZmzZoFOzQgKFgHEAAAwGaYAwgAAGAzJIAAAAA2QwIIAABgMySAAAAANkMCCAAAYDMkgAAAADZDAggAAGAzJIAAAAA2QwIIAABgMySAAAAANkMCCAAAYDMkgAAAADZDAggAAGAzJIAAAAA2QwIIAABgMySAAAAANkMCCAAAYDMkgAAAADZDAggAAGAzJIAAAAA2QwIIAABgMySAAAAANkMCCAAAYDMkgAAAADZDAggAAGAzJIAAAAA2QwIIAABgMySAAAAANkMCCAAAYDMkgAAAADZDAggAAGAz/x97aSPaLF5IxgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAf5UlEQVR4nO3df2zU93348ZcNsVlW24QyDE7MULI0mZvUnsBYpI0WKmuEILKGda3aKnNoRzbV6aa6mgbrOrolK6hpI5TlNrR0iE2qFtppRVFJu65uEE1GAyPzlM1dJhazkWR2xrLY4Ggm2J/vH1Osr8EQfOb88fn9eEj3hz/++O517x65Zz/341ORZVkWAAAkozLvAQAAmFkCEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMfPzHqCcjY2Nxauvvho1NTVRUVGR9zgAwGXIsixOnz4dDQ0NUVmZ5rEwATgNr776ajQ2NuY9BgBQhJMnT8Z1112X9xi5EIDTUFNTExH/9wCqra3NeRoA4HIMDQ1FY2Pj+PN4igTgNLz9sm9tba0ABIAyk/Lbt9J84RsAIGECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxSQfgG2+8EatWrYqWlpa45ZZb4vHHH897JACAkkv6VHA1NTVx6NChuPrqq2N4eDhuueWW2LRpU7z73e/OezQAgJJJ+gjgvHnz4uqrr46IiJGRkciyLLIsy3kqAIDSKusAPHToUGzcuDEaGhqioqIi9u/ff8E+hUIhVqxYEQsWLIi2trY4cuTIhN+/8cYb0dzcHNddd1381m/9VixevHiGpgcAyEdZvwQ8PDwczc3N8clPfjI2bdp0we/37dsXXV1dsXv37mhra4tdu3bFunXr4sUXX4wlS5ZERMTChQvjH//xH2NgYCA2bdoUH/7wh6O+vn6m7wo5WrH1wDvuc2LnhhmYBABmRlkfAVy/fn089NBDcc8990z6+0ceeSS2bNkSmzdvjqampti9e3dcffXVsWfPngv2ra+vj+bm5vjhD3940dsbGRmJoaGhCRcAgHJT1gF4KWfPno1jx45Fe3v7+LbKyspob2+Pw4cPR0TEwMBAnD59OiIiBgcH49ChQ3HTTTdd9Dp37NgRdXV145fGxsbS3gkAgBKYswF46tSpGB0dveDl3Pr6+ujv74+IiH//93+P22+/PZqbm+P222+Pz3zmM3Hrrbde9Dq3bdsWg4OD45eTJ0+W9D4AAJRCWb8HcLpWr14dPT09l71/dXV1VFdXl24gZq3z3yfoPYEAlLM5ewRw8eLFMW/evBgYGJiwfWBgIJYuXZrTVAAA+ZuzAVhVVRUrV66M7u7u8W1jY2PR3d0da9asyXEyAIB8lfVLwGfOnInjx4+P/9zX1xc9PT2xaNGiWL58eXR1dUVHR0esWrUqVq9eHbt27Yrh4eHYvHnztG63UChEoVCI0dHR6d4FZtjlfOULAMx1FVkZn/ri4MGDsXbt2gu2d3R0xN69eyMi4rHHHouHH344+vv7o6WlJR599NFoa2u7Irc/NDQUdXV1MTg4GLW1tVfkOimtKxWA3gMIUL48f5d5AObNA6j8CEAAPH/P4fcAAgAwubJ+DyC8E+/5A4ALCUAowmRh6WVhAMqFl4CLUCgUoqmpKVpbW/MeBQBgygRgETo7O6O3tzeOHj2a9ygAAFMmAAEAEiMAAQASIwABABIjAAEAEiMAi+BTwABAOROARfApYACgnAlAAIDEOBMIc4bTvgHA5RGAcIWcH6BODQfAbOUlYACAxAhAAIDECMAi+BoYAKCcCcAi+BoYAKCcCUAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDEOBUclMhk5yZ2ejgAZgNHAIvgi6ABgHImAIvgi6ABgHImAAEAEiMAAQAS40MglK3JPmQBALwzRwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwCL4FRwAEA5q8iyLMt7iHI1NDQUdXV1MTg4GLW1tXmPk5y58D2AJ3ZuyHsEgOR4/nYEEAAgOQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAx8/MeAC7HXPjSZwCYLRwBBABIjAAEAEiMAAQASIwALEKhUIimpqZobW3NexQAgCmryLIsy3uIcjU0NBR1dXUxODgYtbW1eY8zp6X0IZATOzfkPQLAnOb52xFAAIDkCEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAjAIhQKhWhqaorW1ta8RwEAmLKKLMuyvIcoV0NDQ1FXVxeDg4NRW1ub9zhzyoqtB/IeITcndm7IewSAOc3ztyOAAADJmZ/3AMBE5x/9dEQQgCvNEUAAgMQIQACAxAhAAIDECEAAgMQIQACAxPgUMLlL+Tv/ACAPjgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkxplAYJab7EwpJ3ZuyGESAOYKRwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwABABKTdACePHky7rjjjmhqaor3ve998c1vfjPvkQAASi7pL4KeP39+7Nq1K1paWqK/vz9WrlwZd911V/zkT/5k3qMBAJRM0gG4bNmyWLZsWURELF26NBYvXhyvv/66AAQA5rSyfgn40KFDsXHjxmhoaIiKiorYv3//BfsUCoVYsWJFLFiwINra2uLIkSOTXtexY8didHQ0GhsbSzw1AEC+yvoI4PDwcDQ3N8cnP/nJ2LRp0wW/37dvX3R1dcXu3bujra0tdu3aFevWrYsXX3wxlixZMr7f66+/Hr/yK78Sjz/++EyOn6zJzm0LAMycsg7A9evXx/r16y/6+0ceeSS2bNkSmzdvjoiI3bt3x4EDB2LPnj2xdevWiIgYGRmJD33oQ7F169a47bbbLnl7IyMjMTIyMv7z0NDQFbgXAAAzq6xfAr6Us2fPxrFjx6K9vX18W2VlZbS3t8fhw4cjIiLLsrjvvvvigx/8YNx7773veJ07duyIurq68YuXiwGAcjRnA/DUqVMxOjoa9fX1E7bX19dHf39/REQ8++yzsW/fvti/f3+0tLRES0tLvPDCCxe9zm3btsXg4OD45eTJkyW9DwAApVDWLwFP1wc+8IEYGxu77P2rq6ujurq6hBMBAJTenD0CuHjx4pg3b14MDAxM2D4wMBBLly7NaSoAgPzN2QCsqqqKlStXRnd39/i2sbGx6O7ujjVr1uQ4GQBAvsr6JeAzZ87E8ePHx3/u6+uLnp6eWLRoUSxfvjy6urqio6MjVq1aFatXr45du3bF8PDw+KeCi1UoFKJQKMTo6Oh07wIAwIyryLIsy3uIYh08eDDWrl17wfaOjo7Yu3dvREQ89thj8fDDD0d/f3+0tLTEo48+Gm1tbVfk9oeGhqKuri4GBwejtrb2ilxnCnwP4PSd2Lkh7xEAypbn7zIPwLx5ABVHAE6fAAQonufvOfweQAAAJicAAQASIwABABIjAItQKBSiqakpWltb8x4FAGDKBGAROjs7o7e3N44ePZr3KAAAU1bW3wPI7OcTvwAw+zgCCACQGAEIAJAYAQgAkBgBCACQGB8CKUKhUIhCoRCjo6N5j0Kizv9wjVPDATAVjgAWwdfAAADlTAACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAFqFQKERTU1O0trbmPQoAwJQJwCL4GhgAoJwJQACAxAhAAIDECEAAgMQIQACAxMzPewDmlhVbD+Q9AgDwDhwBBABIjAAEAEiMACyCL4IGAMqZACyCL4IGAMqZAAQASIwABABIjAAEAEiMAAQASIwABABIjAAEAEiMU8HBHDDZKfhO7NyQwyQAlANHAAEAEiMAAQASIwCL4FRwAEA5E4BFcCo4AKCcCUAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDECMAiFAqFaGpqitbW1rxHAQCYMgFYhM7Ozujt7Y2jR4/mPQoAwJQJQACAxAhAAIDECEAAgMQIQACAxAhAAIDEzM97AKA0Vmw9MOHnEzs35DQJALONI4AAAIkRgAAAiRGAAACJEYAAAIkRgAAAiRGAAACJEYAAAIkRgAAAiRGAAACJcSYQLpszSwDA3OAIIABAYgRgEQqFQjQ1NUVra2veowAATJkALEJnZ2f09vbG0aNH8x4FAGDKBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBinAkEEnH+mVwinM0FIFWOAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkxqeAKdpknyoFAGY/RwABABIjAAEAEiMAAQAS4z2AkLDz38fpzCAAaXAEEAAgMQIQACAxAhAAIDECEAAgMQIQACAxAhAAIDG+BoZJOc0bAMxdjgACACRGAAIAJCb5ALznnnvimmuuiQ9/+MN5jwIAMCOSD8Df/M3fjL/4i7/IewwAgBmTfADecccdUVNTk/cYAAAzpqwD8NChQ7Fx48ZoaGiIioqK2L9//wX7FAqFWLFiRSxYsCDa2triyJEjMz8oAMAsUtYBODw8HM3NzVEoFCb9/b59+6Krqyu2b98ezz//fDQ3N8e6devitddem+FJAQBmj7L+HsD169fH+vXrL/r7Rx55JLZs2RKbN2+OiIjdu3fHgQMHYs+ePbF169Yp397IyEiMjIyM/zw0NDT1oQEAclbWRwAv5ezZs3Hs2LFob28f31ZZWRnt7e1x+PDhoq5zx44dUVdXN35pbGy8UuMCAMyYORuAp06ditHR0aivr5+wvb6+Pvr7+8d/bm9vj1/+5V+Op556Kq677rpLxuG2bdticHBw/HLy5MmSzQ8AUCpl/RLwlfD973//svetrq6O6urqEk4DAFB6c/YI4OLFi2PevHkxMDAwYfvAwEAsXbo0p6kAAPI3ZwOwqqoqVq5cGd3d3ePbxsbGoru7O9asWZPjZAAA+Srrl4DPnDkTx48fH/+5r68venp6YtGiRbF8+fLo6uqKjo6OWLVqVaxevTp27doVw8PD458KLlahUIhCoRCjo6PTvQsAADOuIsuyLO8hinXw4MFYu3btBds7Ojpi7969ERHx2GOPxcMPPxz9/f3R0tISjz76aLS1tV2R2x8aGoq6uroYHByM2traK3Kds8WKrQfyHoEcnNi5Ie8RAEpuLj9/X66yDsC8zeUHkABMkwAEUjCXn78v15x9DyAAAJMTgAAAiRGAAACJEYBFKBQK0dTUFK2trXmPAgAwZQKwCJ2dndHb2xtHjx7NexQAgCkTgAAAiRGAAACJEYAAAIkRgAAAiRGARfApYACgnAnAIvgUMABQzgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgRgEXwPIABQzgRgEXwPIABQzgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgRgEXwPIABQzgRgEXwPIABQzgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYubnPUA5KhQKUSgUYnR0NO9RLsuKrQcm/Hxi54Z33Ic0Xc7jYLLHDwDlxRHAIjgVHABQzgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYubnPUA5KhQKUSgUYnR0NO9RirJi64G8RwAAcuQIYBE6Ozujt7c3jh49mvcoAABTJgABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASIwABABIjAAEAEiMAAQASMz/vAcpRoVCIQqEQo6OjJb2dFVsPTPj5xM4NJb09uBznPy4jLnxsTrbPO/0NADPHEcAidHZ2Rm9vbxw9ejTvUQAApkwAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkRgACACRGAAIAJEYAAgAkJvkA/Pa3vx033XRT3HjjjfG1r30t73EAAEpuft4D5OncuXPR1dUVTz/9dNTV1cXKlSvjnnvuiXe/+915jwYAUDJJHwE8cuRIvPe9741rr7023vWud8X69evje9/7Xt5jAQCUVFkH4KFDh2Ljxo3R0NAQFRUVsX///gv2KRQKsWLFiliwYEG0tbXFkSNHxn/36quvxrXXXjv+87XXXhuvvPLKTIwOAJCbsg7A4eHhaG5ujkKhMOnv9+3bF11dXbF9+/Z4/vnno7m5OdatWxevvfbaDE8KADB7lHUArl+/Ph566KG45557Jv39I488Elu2bInNmzdHU1NT7N69O66++urYs2dPREQ0NDRMOOL3yiuvRENDw0Vvb2RkJIaGhiZcAADKzZz9EMjZs2fj2LFjsW3btvFtlZWV0d7eHocPH46IiNWrV8c//dM/xSuvvBJ1dXXxne98J77whS9c9Dp37NgRv//7v1/y2S9mxdYDF2w7sXNDDpPA9JXy8Xz+def97+RK3deZ/G+A/96Uxmx7bM421mfmlPURwEs5depUjI6ORn19/YTt9fX10d/fHxER8+fPj69+9auxdu3aaGlpic997nOX/ATwtm3bYnBwcPxy8uTJkt4HAIBSmLNHAC/X3XffHXffffdl7VtdXR3V1dUlnggAoLTm7BHAxYsXx7x582JgYGDC9oGBgVi6dGlOUwEA5G/OBmBVVVWsXLkyuru7x7eNjY1Fd3d3rFmzJsfJAADyVdYvAZ85cyaOHz8+/nNfX1/09PTEokWLYvny5dHV1RUdHR2xatWqWL16dezatSuGh4dj8+bNOU4NAJCvsg7Av//7v4+1a9eO/9zV1RURER0dHbF379746Ec/Gv/1X/8Vv/d7vxf9/f3R0tIS3/3udy/4YMhUFQqFKBQKMTo6Oq3rAQDIQ1kH4B133BFZll1ynwceeCAeeOCBK3q7nZ2d0dnZGUNDQ1FXV3dFrxsAoNTm7HsAAQCYnAAEAEiMAAQASIwALEKhUIimpqZobW3NexQAgCkTgEXo7OyM3t7eOHr0aN6jAABMmQAEAEiMAAQASIwABABITFl/EXTe3v4S6qGhoZJc/9jIm++4z+Xc9uVcD0zH+Y/DYh9zV+rf0vm3X6p/o5drsvUoZqYrdT2z7bZSMtsem7PNTK3P29f7TieTmMsqspTv/TS9/PLL0djYmPcYAEARTp48Gdddd13eY+RCAE7D2NhYvPrqq1FTUxMVFRV5jxNDQ0PR2NgYJ0+ejNra2rzHKUvWcHqs3/RZw+mxftOXwhpmWRanT5+OhoaGqKxM891wXgKehsrKyln5/xxqa2vn7D/amWINp8f6TZ81nB7rN31zfQ3r6uryHiFXaWYvAEDCBCAAQGIE4BxSXV0d27dvj+rq6rxHKVvWcHqs3/RZw+mxftNnDdPgQyAAAIlxBBAAIDECEAAgMQIQACAxAhAAIDECsMy9/vrr8YlPfCJqa2tj4cKF8alPfSrOnDlzyf0/85nPxE033RQ/8RM/EcuXL4/f+I3fiMHBwRmcOj+FQiFWrFgRCxYsiLa2tjhy5Mgl9//mN78ZN998cyxYsCBuvfXWeOqpp2Zo0tlrKmv4+OOPx+233x7XXHNNXHPNNdHe3v6Oa56CqT4O3/bEE09ERUVFfOhDHyrtgLPcVNfvjTfeiM7Ozli2bFlUV1fHe97znuT/LU91DXft2jX+vNHY2Bif/exn43//939naFpKIqOs3XnnnVlzc3P2ox/9KPvhD3+Y/czP/Ez2sY997KL7v/DCC9mmTZuyJ598Mjt+/HjW3d2d3Xjjjdkv/dIvzeDU+XjiiSeyqqqqbM+ePdk///M/Z1u2bMkWLlyYDQwMTLr/s88+m82bNy/78pe/nPX29ma/+7u/m1111VXZCy+8MMOTzx5TXcOPf/zjWaFQyP7hH/4h+/GPf5zdd999WV1dXfbyyy/P8OSzx1TX8G19fX3Ztddem91+++3ZL/7iL87MsLPQVNdvZGQkW7VqVXbXXXdlzzzzTNbX15cdPHgw6+npmeHJZ4+pruHXv/71rLq6Ovv617+e9fX1ZX/zN3+TLVu2LPvsZz87w5NzJQnAMtbb25tFRHb06NHxbd/5zneyioqK7JVXXrns6/nGN76RVVVVZW+99VYpxpw1Vq9enXV2do7/PDo6mjU0NGQ7duyYdP+PfOQj2YYNGyZsa2try37t136tpHPOZlNdw/OdO3cuq6mpyf78z/+8VCPOesWs4blz57Lbbrst+9rXvpZ1dHQkHYBTXb8/+ZM/ya6//vrs7NmzMzXirDfVNezs7Mw++MEPTtjW1dWVvf/97y/pnJSWl4DL2OHDh2PhwoWxatWq8W3t7e1RWVkZzz333GVfz+DgYNTW1sb8+XP31NBnz56NY8eORXt7+/i2ysrKaG9vj8OHD0/6N4cPH56wf0TEunXrLrr/XFfMGp7vzTffjLfeeisWLVpUqjFntWLX8A/+4A9iyZIl8alPfWomxpy1ilm/J598MtasWROdnZ1RX18ft9xyS3zpS1+K0dHRmRp7VilmDW+77bY4duzY+MvEL730Ujz11FNx1113zcjMlMbcfcZPQH9/fyxZsmTCtvnz58eiRYuiv7//sq7j1KlT8eCDD8b9999fihFnjVOnTsXo6GjU19dP2F5fXx//8i//Munf9Pf3T7r/5a7tXFPMGp7vt3/7t6OhoeGCsE5FMWv4zDPPxJ/92Z9FT0/PDEw4uxWzfi+99FL84Ac/iE984hPx1FNPxfHjx+PTn/50vPXWW7F9+/aZGHtWKWYNP/7xj8epU6fiAx/4QGRZFufOnYtf//Vfj9/5nd+ZiZEpEUcAZ6GtW7dGRUXFJS+X+4R7KUNDQ7Fhw4ZoamqKL37xi9MfHC5h586d8cQTT8S3vvWtWLBgQd7jlIXTp0/HvffeG48//ngsXrw473HK0tjYWCxZsiT+9E//NFauXBkf/ehH4/Of/3zs3r0779HKxsGDB+NLX/pS/PEf/3E8//zz8dd//ddx4MCBePDBB/MejWlwBHAW+tznPhf33XffJfe5/vrrY+nSpfHaa69N2H7u3Ll4/fXXY+nSpZf8+9OnT8edd94ZNTU18a1vfSuuuuqq6Y49qy1evDjmzZsXAwMDE7YPDAxcdK2WLl06pf3numLW8G1f+cpXYufOnfH9738/3ve+95VyzFltqmv4b//2b3HixInYuHHj+LaxsbGI+L+j/S+++GLccMMNpR16FinmMbhs2bK46qqrYt68eePbfvZnfzb6+/vj7NmzUVVVVdKZZ5ti1vALX/hC3HvvvfGrv/qrERFx6623xvDwcNx///3x+c9/PiorHUsqR/5Xm4V+6qd+Km6++eZLXqqqqmLNmjXxxhtvxLFjx8b/9gc/+EGMjY1FW1vbRa9/aGgofuEXfiGqqqriySefTOJoTFVVVaxcuTK6u7vHt42NjUV3d3esWbNm0r9Zs2bNhP0jIv72b//2ovvPdcWsYUTEl7/85XjwwQfju9/97oT3q6Zoqmt48803xwsvvBA9PT3jl7vvvjvWrl0bPT090djYOJPj566Yx+D73//+OH78+Hg4R0T867/+ayxbtiy5+Isobg3ffPPNCyLv7aDOsqx0w1JaeX8Khem58847s5/7uZ/LnnvuueyZZ57JbrzxxglfA/Pyyy9nN910U/bcc89lWZZlg4ODWVtbW3brrbdmx48fz/7zP/9z/HLu3Lm87saMeOKJJ7Lq6ups7969WW9vb3b//fdnCxcuzPr7+7Msy7J7770327p16/j+zz77bDZ//vzsK1/5SvbjH/842759u6+BmeIa7ty5M6uqqsr+6q/+asJj7fTp03ndhdxNdQ3Pl/qngKe6fv/xH/+R1dTUZA888ED24osvZt/+9rezJUuWZA899FBedyF3U13D7du3ZzU1Ndlf/uVfZi+99FL2ve99L7vhhhuyj3zkI3ndBa4AAVjm/vu//zv72Mc+lr3rXe/Kamtrs82bN094cu3r68siInv66aezLMuyp59+OouISS99fX353IkZ9Ed/9EfZ8uXLs6qqqmz16tXZj370o/Hf/fzP/3zW0dExYf9vfOMb2Xve856sqqoqe+9735sdOHBghieefaayhj/90z896WNt+/btMz/4LDLVx+H/L/UAzLKpr9/f/d3fZW1tbVl1dXV2/fXXZ3/4h3845/8P7zuZyhq+9dZb2Re/+MXshhtuyBYsWJA1NjZmn/70p7P/+Z//mfnBuWIqsszxWwCAlHgPIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYgQgAEBiBCAAQGIEIABAYv4f8JhbMNXKcYsAAAAASUVORK5CYII=",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "c = ax.imshow(correlation_r_sq, cmap=\"viridis\")\n",
    "ax.set_title(\"correlation between cosine sim and correlation\")\n",
    "fig.colorbar(c)\n",
    "plt.savefig(\"correlation_r_sq.png\")\n",
    "\n",
    "plt.close(fig)\n",
    "\n",
    "from IPython.display import Image, display\n",
    "display(Image(\"correlation_r_sq.png\"))\n",
    "\n",
    "idxs = np.random.choice(covariances[(0, 1)].flatten().shape[0], size=10000, replace=False)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist(cs_scores[(0, 1)].flatten()[idxs].cpu(), bins=100)\n",
    "ax.set_yscale(\"log\")\n",
    "plt.savefig(\"correlation_hist.png\")\n",
    "plt.close(fig)\n",
    "\n",
    "display(Image(\"correlation_hist.png\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't convert cuda:3 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[70], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m cs_max, cs_max_idxs \u001b[39m=\u001b[39m max_cosine_sim(filtered_dicts[layer_0][\u001b[39m0\u001b[39m], filtered_dicts[layer_1][\u001b[39m0\u001b[39m])\n\u001b[1;32m      6\u001b[0m fig, ax \u001b[39m=\u001b[39m plt\u001b[39m.\u001b[39msubplots()\n\u001b[0;32m----> 7\u001b[0m ax\u001b[39m.\u001b[39;49mhist(cs_max, bins\u001b[39m=\u001b[39;49m\u001b[39m100\u001b[39;49m)\n\u001b[1;32m      8\u001b[0m ax\u001b[39m.\u001b[39mset_xlabel(\u001b[39m\"\u001b[39m\u001b[39mMax Cosine Similarity\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m      9\u001b[0m ax\u001b[39m.\u001b[39mset_ylabel(\u001b[39m\"\u001b[39m\u001b[39mCount\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/sparse_coding_aidan/.env/lib/python3.11/site-packages/matplotlib/__init__.py:1442\u001b[0m, in \u001b[0;36m_preprocess_data.<locals>.inner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1439\u001b[0m \u001b[39m@functools\u001b[39m\u001b[39m.\u001b[39mwraps(func)\n\u001b[1;32m   1440\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39minner\u001b[39m(ax, \u001b[39m*\u001b[39margs, data\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m   1441\u001b[0m     \u001b[39mif\u001b[39;00m data \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 1442\u001b[0m         \u001b[39mreturn\u001b[39;00m func(ax, \u001b[39m*\u001b[39;49m\u001b[39mmap\u001b[39;49m(sanitize_sequence, args), \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1444\u001b[0m     bound \u001b[39m=\u001b[39m new_sig\u001b[39m.\u001b[39mbind(ax, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m   1445\u001b[0m     auto_label \u001b[39m=\u001b[39m (bound\u001b[39m.\u001b[39marguments\u001b[39m.\u001b[39mget(label_namer)\n\u001b[1;32m   1446\u001b[0m                   \u001b[39mor\u001b[39;00m bound\u001b[39m.\u001b[39mkwargs\u001b[39m.\u001b[39mget(label_namer))\n",
      "File \u001b[0;32m~/sparse_coding_aidan/.env/lib/python3.11/site-packages/matplotlib/axes/_axes.py:6703\u001b[0m, in \u001b[0;36mAxes.hist\u001b[0;34m(self, x, bins, range, density, weights, cumulative, bottom, histtype, align, orientation, rwidth, log, color, label, stacked, **kwargs)\u001b[0m\n\u001b[1;32m   6700\u001b[0m     stacked \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m   6702\u001b[0m \u001b[39m# Massage 'x' for processing.\u001b[39;00m\n\u001b[0;32m-> 6703\u001b[0m x \u001b[39m=\u001b[39m cbook\u001b[39m.\u001b[39;49m_reshape_2D(x, \u001b[39m'\u001b[39;49m\u001b[39mx\u001b[39;49m\u001b[39m'\u001b[39;49m)\n\u001b[1;32m   6704\u001b[0m nx \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39m(x)  \u001b[39m# number of datasets\u001b[39;00m\n\u001b[1;32m   6706\u001b[0m \u001b[39m# Process unit information.  _process_unit_info sets the unit and\u001b[39;00m\n\u001b[1;32m   6707\u001b[0m \u001b[39m# converts the first dataset; then we convert each following dataset\u001b[39;00m\n\u001b[1;32m   6708\u001b[0m \u001b[39m# one at a time.\u001b[39;00m\n",
      "File \u001b[0;32m~/sparse_coding_aidan/.env/lib/python3.11/site-packages/matplotlib/cbook/__init__.py:1399\u001b[0m, in \u001b[0;36m_reshape_2D\u001b[0;34m(X, name)\u001b[0m\n\u001b[1;32m   1397\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   1398\u001b[0m         is_1d \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[0;32m-> 1399\u001b[0m xi \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39masanyarray(xi)\n\u001b[1;32m   1400\u001b[0m nd \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mndim(xi)\n\u001b[1;32m   1401\u001b[0m \u001b[39mif\u001b[39;00m nd \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n",
      "File \u001b[0;32m~/sparse_coding_aidan/.env/lib/python3.11/site-packages/torch/_tensor.py:970\u001b[0m, in \u001b[0;36mTensor.__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m    968\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(Tensor\u001b[39m.\u001b[39m__array__, (\u001b[39mself\u001b[39m,), \u001b[39mself\u001b[39m, dtype\u001b[39m=\u001b[39mdtype)\n\u001b[1;32m    969\u001b[0m \u001b[39mif\u001b[39;00m dtype \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 970\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnumpy()\n\u001b[1;32m    971\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    972\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnumpy()\u001b[39m.\u001b[39mastype(dtype, copy\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n",
      "\u001b[0;31mTypeError\u001b[0m: can't convert cuda:3 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
     ]
    }
   ],
   "source": [
    "layer_0 = 0\n",
    "layer_1 = 1\n",
    "\n",
    "cs_max, cs_max_idxs = max_cosine_sim(filtered_dicts[layer_0][0], filtered_dicts[layer_1][0])\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist(cs_max.cpu(), bins=100)\n",
    "ax.set_xlabel(\"Max Cosine Similarity\")\n",
    "ax.set_ylabel(\"Count\")\n",
    "plt.savefig(\"graphs/max_cosine_sim.png\")\n",
    "\n",
    "plt.close(fig)\n",
    "\n",
    "display(Image(\"graphs/max_cosine_sim.png\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
